import re
import base64
import requests
import time
from scrapy import Selector
import pymongo

myclient = pymongo.MongoClient('mongodb://127.0.0.1:27017')
mydb = myclient["darenji_anchors"]
mycol = mydb["anchors"]
pattern = re.compile(r"\d+\.?\d*")

def main(begin, end):

    id = begin # 16000

    for item in range(50000):
        id = id + 1
        encodestr = base64.b64encode(str(id).encode('utf-8'))
        url = "http://www.darenji.com/detail/%s.html" % str(encodestr, 'utf-8')

        html = requests.get(url)
        html_text = html.text
        sel = Selector(text=html_text)  # 初始化

        anchor_name = sel.css(".wxDetail-name .fs22 a::attr(title)")
        anchor_name_res = anchor_name.extract_first(default='')

        original_name = sel.css(".original::text")
        original_name_res = original_name.extract_first(default='')

        if anchor_name_res:

            anchor_avatar = sel.css(".wxDetail-img img::attr(src)")
            anchor_avatar = anchor_avatar.extract_first(default='')

            anchor_id = sel.css(".wxcode-down img::attr(src)")
            anchor_id = anchor_id.extract_first(default='')
            anchor_id = re.findall(pattern, anchor_id)[0]

            fans_num = sel.css(".info-li div:nth-of-type(2) p::text")
            fans_num = fans_num[2].extract().strip()
            fans_num_type = fans_num.find("万") != -1
            fans_num = int(re.findall(pattern, fans_num)[0])
            if fans_num_type:
                fans_num = fans_num * 10000

            mydict = {
                "drjId": id,
                "anchorName": anchor_name_res,
                "originalName": original_name_res,
                "anchorAvatar": anchor_avatar,
                "fansNumber": fans_num,
                "anchorId": anchor_id
            }
            mycol.insert_one(mydict)
            print(id)

        if id >= end:
            break

        time.sleep(1)

    print('end' + str(id))

if __name__ == "__main__":
    main(165000, 170000)
